This notebook processes level 2 images through the calwebb_image3 skymatch and resample steps and examines outputs for different sky method parameters.
1) Set up data path and image list file.
2) Set up association files.
3) Modify average backgrould level of input images.
4) Run skymatch step on images.
5) Run skymatch and resample for each skymethod parameter (local, global, match, global+match).
6) Testing other parameters (nclip, usigma, lsigma, lower, upper, skystat). (Still TBD.)
These steps are set up with simulated MIRI F560W data of the LMC astrometric field. The notebook removes the astrometric field and replaces the field with noise images to help the pipeline testers have a better visualization of what the pipeline is doing with each set of parameters.
This notebook has also been modified to test how skymatch and resample work together (with subtract=True indicating that the subtraction is done in the skymatch step and subtract=False indicating that the subtraction is done in the resample step). There are displays of the combined noise image to see how well the backgrounds were subtracted in the full combined image.
The pipeline documentation can be found here: https://jwst-pipeline.readthedocs.io/en/latest/jwst/skymatch/README.html
The pipeline code is available on GitHub: https://github.com/spacetelescope/jwst/tree/master/jwst/skymatch
Authors: T. Temim and M. Cracraft Last modified: 01/18/2022
# Create a temporary directory to hold notebook output, and change the working directory to that directory.
from tempfile import TemporaryDirectory
import os
data_dir = TemporaryDirectory()
os.chdir(data_dir.name)
# Set up CRDS options
import os
if "CRDS_CACHE_TYPE" in os.environ:
if os.environ['CRDS_CACHE_TYPE'] == 'local':
os.environ['CRDS_PATH'] = os.path.join(os.environ['HOME'], 'crds', 'cache')
elif os.path.isdir(os.environ['CRDS_CACHE_TYPE']):
os.environ['CRDS_PATH'] = os.environ['CRDS_CACHE_TYPE']
The following packages are needed to run this notebook:
import pytest
from astropy.io import fits
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
import jwst
import json
from jwst.skymatch import skymatch
from jwst.resample import ResampleStep
from jwst.pipeline import Image3Pipeline
from jwst.skymatch.skyimage import SkyImage
from jwst.associations.lib.rules_level3_base import DMS_Level3_Base
from jwst.associations import asn_from_list
from jwst import datamodels
from jwst.datamodels import ImageModel
from matplotlib import rcParams
from ci_watson.artifactory_helpers import get_bigdata
jwst.__version__
'1.3.2'
input_file1 = get_bigdata('jwst_validation_notebooks',
'validation_data',
'outlier_detection',
'outlier_detection_miri_test',
'det_image_seq1_MIRIMAGE_F560Wexp1_cal.fits')
input_file2 = get_bigdata('jwst_validation_notebooks',
'validation_data',
'outlier_detection',
'outlier_detection_miri_test',
'det_image_seq2_MIRIMAGE_F560Wexp1_cal.fits')
input_file3 = get_bigdata('jwst_validation_notebooks',
'validation_data',
'outlier_detection',
'outlier_detection_miri_test',
'det_image_seq3_MIRIMAGE_F560Wexp1_cal.fits')
input_file4 = get_bigdata('jwst_validation_notebooks',
'validation_data',
'outlier_detection',
'outlier_detection_miri_test',
'det_image_seq4_MIRIMAGE_F560Wexp1_cal.fits')
input_files=[]
input_files=[input_file1,input_file2,input_file3,input_file4]
imlist1=['det_image_seq1_MIRIMAGE_F560Wexp1_cal.fits','det_image_seq2_MIRIMAGE_F560Wexp1_cal.fits','det_image_seq3_MIRIMAGE_F560Wexp1_cal.fits','det_image_seq4_MIRIMAGE_F560Wexp1_cal.fits']
# Look at one image
im_file = ImageModel('det_image_seq1_MIRIMAGE_F560Wexp1_cal.fits')
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=5)
plt.colorbar()
<matplotlib.colorbar.Colorbar at 0x7efd40e635b0>
The level three pipeline relies on an association file to specify which files are to be combined and provide the output file name.
# use asn_from_list to create association table
cal_list=imlist1
asn = asn_from_list.asn_from_list(cal_list, rule=DMS_Level3_Base, product_name='skymatch_combined.fits')
# use this if you need to add non'science' exposure types
#asn['products'][0]['members'][1]['exptype'] = 'background'
#asn['products'][0]['members'][2]['exptype'] = 'sourcecat'
# dump association table to a .json file for use in image3
with open('skymatch_asnfile.json', 'w') as fp:
fp.write(asn.dump()[1])
skymatch_json_file='skymatch_asnfile.json'
json_file = skymatch_json_file
file_list = []
file_list2 = []
with open(json_file) as json_data:
d = json.load(json_data)
members = d['products'][0]['members']
for item in np.arange(0,len(members)):
file_list.append(members[item]['expname'])
file_list2.append(members[item]['expname'][:-5]+"_skymatch.fits")
asn2 = asn_from_list.asn_from_list(file_list2, rule=DMS_Level3_Base, product_name=d['products'][0]['name'])
# use this if you need to add non'science' exposure types
#asn['products'][0]['members'][1]['exptype'] = 'background'
#asn['products'][0]['members'][2]['exptype'] = 'sourcecat'
# dump association table to a .json file for use in image3
with open('skymatch_asnfile2.json', 'w') as fp:
fp.write(asn2.dump()[1])
skymatch_json_file2='skymatch_asnfile2.json'
infile01_1 = input_files[0]
infile01_2 = input_files[1]
infile02_1 = input_files[2]
infile02_2 = input_files[3]
img01_1 = datamodels.open(infile01_1)
img01_2 = datamodels.open(infile01_2)
img02_1 = datamodels.open(infile02_1)
img02_2 = datamodels.open(infile02_2)
data01_1 = img01_1.data
data01_2 = img01_2.data
data02_1 = img02_1.data
data02_2 = img02_2.data
data01_1[data01_1<=0.3]=np.nan
data01_2[data01_2<=0.3]=np.nan
data02_1[data02_1<=0.3]=np.nan
data02_2[data02_2<=0.3]=np.nan
data01_1_orig = np.copy(img01_1.data)
data01_2_orig = np.copy(img01_2.data)
data02_1_orig = np.copy(img02_1.data)
data02_2_orig = np.copy(img02_2.data)
# check mean values of background
print('Mean:', np.mean(data01_1_orig[data01_1_orig<=4.0]),' Standard deviation: ',np.std(data01_1_orig[data01_1_orig<=4.0]))
print('Mean:', np.mean(data01_2_orig[data01_2_orig<=4.0]),' Standard deviation: ',np.std(data01_2_orig[data01_2_orig<=4.0]))
print('Mean:', np.mean(data02_1_orig[data02_1_orig<=4.0]),' Standard deviation: ',np.std(data02_1_orig[data02_1_orig<=4.0]))
print('Mean:', np.mean(data02_2_orig[data02_2_orig<=4.0]),' Standard deviation: ',np.std(data02_2_orig[data02_2_orig<=4.0]))
Mean: 0.93090874 Standard deviation: 0.1817127 Mean: 0.9362799 Standard deviation: 0.17500266 Mean: 0.9330347 Standard deviation: 0.17194116 Mean: 0.93216574 Standard deviation: 0.16481434
# creating a background image with specified mean and gaussian noise with sigma = 1.0
bkg_img_noise_neg2 = np.random.normal(-2,2*0.2,data01_1.shape)
bkg_img_noise_2 = np.random.normal(2,2*0.2,data01_1.shape)
bkg_img_noise_3 = np.random.normal(3,3*0.2,data01_1.shape)
bkg_img_noise_4 = np.random.normal(4,4*0.2,data01_1.shape)
bkg_img_noise_5 = np.random.normal(5,5*0.2,data01_1.shape)
bkg_img_noise_7 = np.random.normal(7,7*0.2,data01_1.shape)
Replace the images with a noise image for easier background visualization
# adding the new background with specified mean and gaussian noise (above) to image (This overwrites, not adds.)
img01_1.data=bkg_img_noise_2
img01_2.data=bkg_img_noise_3
img02_1.data=bkg_img_noise_5
img02_2.data=bkg_img_noise_2
# checking the mean and standard deviations of the new background values
print('Mean:',np.nanmean(img01_1.data),' Standard deviation: ',np.nanstd(img01_1.data))
print('Mean:',np.nanmean(img01_2.data),' Standard deviation: ',np.nanstd(img01_2.data))
print('Mean:',np.nanmean(img02_1.data),' Standard deviation: ',np.nanstd(img02_1.data))
print('Mean:',np.nanmean(img02_2.data),' Standard deviation: ',np.nanstd(img02_2.data))
Mean: 1.9998825 Standard deviation: 0.39994636 Mean: 2.99957 Standard deviation: 0.59956974 Mean: 4.9972 Standard deviation: 1.0010241 Mean: 1.9998825 Standard deviation: 0.39994636
img01_1.save(file_list2[0],overwrite=True)
img01_2.save(file_list2[1],overwrite=True)
img02_1.save(file_list2[2],overwrite=True)
img02_2.save(file_list2[3],overwrite=True)
'det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits'
# Look at one image
im_file = ImageModel(file_list2[0])
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
<matplotlib.colorbar.Colorbar at 0x7efd3ad79be0>
Run the pipeline on the data for different sets of parameters
Notes: ‘local’: compute sky background values of each input image or group of images (members of the same “exposure”). A single sky value is computed for each group of images. This method simply computes the mean/median/mode/etc. value of the “sky” separately in each input image. This will resulted in subtracted background being near zero in the combined image, as each image has it's individual background subtracted.
match_down specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True)
'Subtract' specifies whether the computed sky background values are to be subtracted from the images in the skymatch step. (Default = False) Currently, if not done in skymatch, the subtraction is performed in resample.
# skymatch, local, subtract= False, all 4 images used
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='local', subtract=False,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'local' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = False # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-01-28 12:47:24,127 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-01-28 12:47:24,129 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-01-28 12:47:24,132 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-01-28 12:47:24,134 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-01-28 12:47:24,136 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-01-28 12:47:24,137 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-01-28 12:47:24,139 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-01-28 12:47:24,265 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-01-28 12:47:24,272 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'local', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-01-28 12:47:24,395 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-01-28 12:47:24,664 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-01-28 12:47:24,667 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-01-28 12:47:25,178 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-01-28 12:47:25,180 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}
2022-01-28 12:47:25,181 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-01-28 12:47:25,187 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-01-28 12:47:25,291 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-01-28 12:47:25,293 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'local', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-01-28 12:47:25,343 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:47:25,344 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-01-28 12:47:25.343536
2022-01-28 12:47:25,345 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:47:25,345 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'local'
2022-01-28 12:47:25,346 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: OFF
2022-01-28 12:47:25,347 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:47:25,347 - stpipe.Image3Pipeline.skymatch - INFO - ---- Sky values computed per image and/or image groups.
2022-01-28 12:47:25,432 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671
2022-01-28 12:47:25,433 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.97803
2022-01-28 12:47:25,434 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 4.95533
2022-01-28 12:47:25,435 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671
2022-01-28 12:47:25,435 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:47:25,436 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-01-28 12:47:25.435871
2022-01-28 12:47:25,437 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:00.092335
2022-01-28 12:47:25,437 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:47:25,696 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-01-28 12:47:25,942 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-01-28 12:47:26,186 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-01-28 12:47:26,425 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-01-28 12:47:26,426 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-01-28 12:47:26,617 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-01-28 12:47:26,619 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}
2022-01-28 12:47:26,620 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-01-28 12:47:26,626 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-01-28 12:47:26,738 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-01-28 12:47:26,740 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}
2022-01-28 12:47:26,759 - stpipe.Image3Pipeline.resample - INFO - Drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-01-28 12:47:26,896 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-01-28 12:47:27,516 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-01-28 12:47:28,119 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:29,056 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:29,999 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:30,939 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:31,307 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-01-28 12:47:31,891 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:33,004 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:34,144 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:35,322 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:35,786 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-01-28 12:47:36,538 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:37,662 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:38,793 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:39,901 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:40,272 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-01-28 12:47:41,075 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:42,156 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:43,285 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:44,494 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:44,947 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020417355 -0.018624953 0.023511130 0.016629342 359.989210484 0.019639429 359.986116709 -0.015614866
2022-01-28 12:47:45,392 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-01-28 12:47:45,395 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-01-28 12:47:45,742 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1146, 1116) from skymatch_combined_i2d.fits>,).
2022-01-28 12:47:45,745 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-01-28 12:47:45,746 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-01-28 12:47:45,748 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-01-28 12:47:45,750 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
# Check values out of skymatch step. With subtract = False, values should be equal.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('Original and new levels should be equal. If subtract=False, skymatch should not subtract.')
print('Mean: original, new (local)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
Original and new levels should be equal. If subtract=False, skymatch should not subtract. Mean: original, new (local) Mean: 1.9998825 , 1.9998825 Mean: 2.99957 , 2.99957 Mean: 4.9972 , 4.9972 Mean: 1.9998825 , 1.9998825
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('The resample step should subtract the individual backgrounds found in the skymatch step.')
print('Since the local individual background level is subtracted from each image, final expected result is near zero.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
The resample step should subtract the individual backgrounds found in the skymatch step. Since the local individual background level is subtracted from each image, final expected result is near zero. Mean: 0.01989578
<matplotlib.colorbar.Colorbar at 0x7efd39c76c10>
Skymatch step should subtract the background. Resample background level should match the skymatch background level since subtraction has already been done
# skymatch, local, subtract = True, all 4 images used
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='local', subtract=True,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'local' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
#match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = True # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
#pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-01-28 12:47:47,311 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-01-28 12:47:47,313 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-01-28 12:47:47,315 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-01-28 12:47:47,317 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-01-28 12:47:47,319 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-01-28 12:47:47,321 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-01-28 12:47:47,322 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-01-28 12:47:47,602 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-01-28 12:47:47,610 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'local', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-01-28 12:47:47,739 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-01-28 12:47:47,742 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-01-28 12:47:47,744 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-01-28 12:47:48,363 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-01-28 12:47:48,365 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}
2022-01-28 12:47:48,367 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-01-28 12:47:48,373 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-01-28 12:47:48,525 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-01-28 12:47:48,527 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'local', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-01-28 12:47:48,586 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:47:48,587 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-01-28 12:47:48.586427
2022-01-28 12:47:48,588 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:47:48,588 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'local'
2022-01-28 12:47:48,589 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: ON
2022-01-28 12:47:48,589 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:47:48,590 - stpipe.Image3Pipeline.skymatch - INFO - ---- Sky values computed per image and/or image groups.
2022-01-28 12:47:48,677 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671 (old=0, delta=1.98671)
2022-01-28 12:47:48,681 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.97803 (old=0, delta=2.97803)
2022-01-28 12:47:48,683 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 4.95533 (old=0, delta=4.95533)
2022-01-28 12:47:48,686 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671 (old=0, delta=1.98671)
2022-01-28 12:47:48,687 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:47:48,688 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-01-28 12:47:48.687514
2022-01-28 12:47:48,689 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:00.101087
2022-01-28 12:47:48,690 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:47:48,982 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-01-28 12:47:49,258 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-01-28 12:47:49,537 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-01-28 12:47:49,782 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-01-28 12:47:49,783 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-01-28 12:47:49,999 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-01-28 12:47:50,002 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}
2022-01-28 12:47:50,002 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-01-28 12:47:50,009 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-01-28 12:47:50,170 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-01-28 12:47:50,172 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}
2022-01-28 12:47:50,188 - stpipe.Image3Pipeline.resample - INFO - Drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-01-28 12:47:50,338 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-01-28 12:47:50,995 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-01-28 12:47:51,631 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:52,611 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:53,586 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:54,576 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:54,939 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-01-28 12:47:55,568 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:56,565 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:57,539 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:58,516 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:47:58,876 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-01-28 12:47:59,497 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:00,504 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:01,471 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:02,434 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:02,804 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-01-28 12:48:03,433 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:04,412 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:05,397 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:06,376 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:06,761 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020417355 -0.018624953 0.023511130 0.016629342 359.989210484 0.019639429 359.986116709 -0.015614866
2022-01-28 12:48:07,190 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-01-28 12:48:07,191 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-01-28 12:48:07,384 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1146, 1116) from skymatch_combined_i2d.fits>,).
2022-01-28 12:48:07,386 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-01-28 12:48:07,387 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-01-28 12:48:07,389 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-01-28 12:48:07,390 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('With subtract=True, the new value should show that the background value was subtracted.')
print('Mean: original, new (local)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
With subtract=True, the new value should show that the background value was subtracted. Mean: original, new (local) Mean: 1.9998825 , 0.013177731 Mean: 2.99957 , 0.021540029 Mean: 4.9972 , 0.041871786 Mean: 1.9998825 , 0.013177731
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('The background value was subtracted in the skymatch step, no additional subtraction should be done here.')
print('This value should match the values in the new column above.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
The background value was subtracted in the skymatch step, no additional subtraction should be done here. This value should match the values in the new column above. Mean: 0.01989578
<matplotlib.colorbar.Colorbar at 0x7efd39702460>
This calculates an average sky across all images and subtracts that average from all images. There should be a warning for users for this step that makes it clear that this should not be used on images with differences in background levels.
# skymatch, local
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='local', subtract=False,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'global' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
#match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = False # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
#pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-01-28 12:48:08,859 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-01-28 12:48:08,861 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-01-28 12:48:08,864 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-01-28 12:48:08,865 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-01-28 12:48:08,867 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-01-28 12:48:08,869 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-01-28 12:48:08,871 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-01-28 12:48:09,101 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-01-28 12:48:09,107 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-01-28 12:48:09,228 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-01-28 12:48:09,231 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-01-28 12:48:09,234 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-01-28 12:48:09,820 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-01-28 12:48:09,822 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}
2022-01-28 12:48:09,823 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-01-28 12:48:09,829 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-01-28 12:48:09,979 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-01-28 12:48:09,981 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-01-28 12:48:10,040 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:10,041 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-01-28 12:48:10.040417
2022-01-28 12:48:10,042 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:10,042 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'global'
2022-01-28 12:48:10,043 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: OFF
2022-01-28 12:48:10,043 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:10,044 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing "global" sky - smallest sky value across *all* input images.
2022-01-28 12:48:10,128 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:10,129 - stpipe.Image3Pipeline.skymatch - INFO - "Global" sky value correction: 1.9867050353989686 [not converted]
2022-01-28 12:48:10,130 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671
2022-01-28 12:48:10,130 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671
2022-01-28 12:48:10,130 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671
2022-01-28 12:48:10,131 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671
2022-01-28 12:48:10,131 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:10,132 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-01-28 12:48:10.131945
2022-01-28 12:48:10,132 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:00.091528
2022-01-28 12:48:10,133 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:10,416 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-01-28 12:48:10,692 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-01-28 12:48:10,965 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-01-28 12:48:11,208 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-01-28 12:48:11,209 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-01-28 12:48:11,401 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-01-28 12:48:11,404 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}
2022-01-28 12:48:11,405 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-01-28 12:48:11,411 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-01-28 12:48:11,567 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-01-28 12:48:11,569 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}
2022-01-28 12:48:11,587 - stpipe.Image3Pipeline.resample - INFO - Drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-01-28 12:48:11,736 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-01-28 12:48:12,405 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-01-28 12:48:13,013 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:13,981 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:14,957 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:15,974 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:16,339 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-01-28 12:48:16,942 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:17,893 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:18,863 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:19,858 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:20,237 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-01-28 12:48:20,841 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:21,810 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:22,796 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:23,824 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:24,242 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-01-28 12:48:24,869 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:25,892 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:26,901 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:27,901 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:28,292 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020417355 -0.018624953 0.023511130 0.016629342 359.989210484 0.019639429 359.986116709 -0.015614866
2022-01-28 12:48:28,722 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-01-28 12:48:28,723 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-01-28 12:48:28,925 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1146, 1116) from skymatch_combined_i2d.fits>,).
2022-01-28 12:48:28,928 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-01-28 12:48:28,929 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-01-28 12:48:28,931 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-01-28 12:48:28,933 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('There should be no subtraction at this step, with subtraction = False.')
print('Mean: original, new (local)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
There should be no subtraction at this step, with subtraction = False. Mean: original, new (local) Mean: 1.9998825 , 1.9998825 Mean: 2.99957 , 2.99957 Mean: 4.9972 , 4.9972 Mean: 1.9998825 , 1.9998825
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('Global finds a single value for the background, and subtracts that value from all images.')
print('The value listed here should be the average of the subtracted sky values across images.')
print('The combined image will look messy as the same value was subtracted from the image with background = 5')
print('and from the images with background = 2. This shows the differing backgrounds strongly.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
Global finds a single value for the background, and subtracts that value from all images. The value listed here should be the average of the subtracted sky values across images. The combined image will look messy as the same value was subtracted from the image with background = 5 and from the images with background = 2. This shows the differing backgrounds strongly. Mean: 1.0094382
<matplotlib.colorbar.Colorbar at 0x7efd39f87880>
# skymatch, global, subtract = True
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='global', subtract=True,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'global' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
#match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = True # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
#pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-01-28 12:48:30,379 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-01-28 12:48:30,381 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-01-28 12:48:30,383 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-01-28 12:48:30,385 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-01-28 12:48:30,386 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-01-28 12:48:30,388 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-01-28 12:48:30,390 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-01-28 12:48:30,626 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-01-28 12:48:30,632 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-01-28 12:48:30,761 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-01-28 12:48:30,769 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-01-28 12:48:30,772 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-01-28 12:48:31,361 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-01-28 12:48:31,363 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}
2022-01-28 12:48:31,364 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-01-28 12:48:31,370 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-01-28 12:48:31,523 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-01-28 12:48:31,526 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-01-28 12:48:31,581 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:31,582 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-01-28 12:48:31.581203
2022-01-28 12:48:31,583 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:31,584 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'global'
2022-01-28 12:48:31,584 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: ON
2022-01-28 12:48:31,585 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:31,585 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing "global" sky - smallest sky value across *all* input images.
2022-01-28 12:48:31,669 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:31,670 - stpipe.Image3Pipeline.skymatch - INFO - "Global" sky value correction: 1.9867050353989686 [not converted]
2022-01-28 12:48:31,673 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671 (old=0, delta=1.98671)
2022-01-28 12:48:31,675 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671 (old=0, delta=1.98671)
2022-01-28 12:48:31,677 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671 (old=0, delta=1.98671)
2022-01-28 12:48:31,680 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.98671 (old=0, delta=1.98671)
2022-01-28 12:48:31,680 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:31,681 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-01-28 12:48:31.680570
2022-01-28 12:48:31,681 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:00.099367
2022-01-28 12:48:31,682 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:31,966 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-01-28 12:48:32,239 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-01-28 12:48:32,497 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-01-28 12:48:32,749 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-01-28 12:48:32,750 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-01-28 12:48:32,954 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-01-28 12:48:32,956 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}
2022-01-28 12:48:32,956 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-01-28 12:48:32,962 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-01-28 12:48:33,118 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-01-28 12:48:33,119 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}
2022-01-28 12:48:33,148 - stpipe.Image3Pipeline.resample - INFO - Drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-01-28 12:48:33,302 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-01-28 12:48:33,985 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-01-28 12:48:34,611 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:35,651 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:36,654 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:37,764 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:38,144 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-01-28 12:48:38,763 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:39,761 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:40,773 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:41,787 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:42,162 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-01-28 12:48:42,779 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:43,763 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:44,753 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:45,775 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:46,137 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-01-28 12:48:46,757 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:47,762 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:48,754 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:49,751 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:50,154 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020417355 -0.018624953 0.023511130 0.016629342 359.989210484 0.019639429 359.986116709 -0.015614866
2022-01-28 12:48:50,595 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-01-28 12:48:50,597 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-01-28 12:48:50,806 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1146, 1116) from skymatch_combined_i2d.fits>,).
2022-01-28 12:48:50,808 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-01-28 12:48:50,809 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-01-28 12:48:50,811 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-01-28 12:48:50,813 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('Global finds a single value for the background, and subtracts that value from all images.')
print('The values listed here should be the average of the subtracted sky values across images subtracted from each image.')
print('Mean: original, new (global)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
Global finds a single value for the background, and subtracts that value from all images. The values listed here should be the average of the subtracted sky values across images subtracted from each image. Mean: original, new (global) Mean: 1.9998825 , 0.013177731 Mean: 2.99957 , 1.0128647 Mean: 4.9972 , 3.0104957 Mean: 1.9998825 , 0.013177731
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('Global finds a single value for the background, and subtracts that value from all images.')
print('The value listed here should be the average of the subtracted sky values across images.')
print('The combined image will look messy as the same value was subtracted from the image with background = 5')
print('and from the images with background = 2. This shows the differing backgrounds strongly.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
Global finds a single value for the background, and subtracts that value from all images. The value listed here should be the average of the subtracted sky values across images. The combined image will look messy as the same value was subtracted from the image with background = 5 and from the images with background = 2. This shows the differing backgrounds strongly. Mean: 1.0094382
<matplotlib.colorbar.Colorbar at 0x7efd3954e9d0>
Based on whether match_down is set to True or False, Match will calculate the difference between the lowest background level or highest level (respectively), and subtract the difference between the calculated level and the 'matched' level.
This is the preferred default option as it subtracts the differences between the background levels, normalizing all of the backgrounds to a common level (either lowest or highest background), rather than subtracting off all of the background.
# skymatch, match down, subtract = True
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='match', match_down=True,subtract=True,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'match' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = True # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-01-28 12:48:52,244 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-01-28 12:48:52,246 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-01-28 12:48:52,248 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-01-28 12:48:52,250 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-01-28 12:48:52,254 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-01-28 12:48:52,255 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-01-28 12:48:52,257 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-01-28 12:48:52,495 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-01-28 12:48:52,502 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-01-28 12:48:52,624 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-01-28 12:48:52,626 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-01-28 12:48:52,630 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-01-28 12:48:53,226 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-01-28 12:48:53,228 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}
2022-01-28 12:48:53,229 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-01-28 12:48:53,235 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-01-28 12:48:53,393 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-01-28 12:48:53,396 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-01-28 12:48:53,452 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:53,453 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-01-28 12:48:53.452150
2022-01-28 12:48:53,454 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:53,455 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'match'
2022-01-28 12:48:53,455 - stpipe.Image3Pipeline.skymatch - INFO - Sky matching direction: DOWN
2022-01-28 12:48:53,456 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: ON
2022-01-28 12:48:53,457 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:53,458 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing differences in sky values in overlapping regions.
2022-01-28 12:48:54,947 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0 (old=0, delta=0)
2022-01-28 12:48:54,950 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.993763 (old=0, delta=0.993763)
2022-01-28 12:48:54,953 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.98947 (old=0, delta=2.98947)
2022-01-28 12:48:54,956 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.0110644 (old=0, delta=0.0110644)
2022-01-28 12:48:54,956 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:54,957 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-01-28 12:48:54.956753
2022-01-28 12:48:54,957 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:01.504603
2022-01-28 12:48:54,958 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:48:55,268 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-01-28 12:48:55,543 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-01-28 12:48:55,822 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-01-28 12:48:56,123 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-01-28 12:48:56,124 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-01-28 12:48:56,434 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-01-28 12:48:56,437 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}
2022-01-28 12:48:56,438 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-01-28 12:48:56,444 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-01-28 12:48:56,712 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-01-28 12:48:56,714 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}
2022-01-28 12:48:56,732 - stpipe.Image3Pipeline.resample - INFO - Drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-01-28 12:48:56,872 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-01-28 12:48:57,500 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-01-28 12:48:58,254 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:48:59,324 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:00,406 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:01,480 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:01,847 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-01-28 12:49:02,564 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:03,649 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:04,718 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:05,801 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:06,163 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-01-28 12:49:06,867 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:07,934 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:09,100 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:10,130 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:10,500 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-01-28 12:49:11,123 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:12,072 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:13,039 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:13,985 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:14,370 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020417355 -0.018624953 0.023511130 0.016629342 359.989210484 0.019639429 359.986116709 -0.015614866
2022-01-28 12:49:14,794 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-01-28 12:49:14,795 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-01-28 12:49:15,124 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1146, 1116) from skymatch_combined_i2d.fits>,).
2022-01-28 12:49:15,126 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-01-28 12:49:15,127 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-01-28 12:49:15,129 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-01-28 12:49:15,130 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('There should be no subtraction at this step, with subtraction = False.')
print('Mean: original, new (match down)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
There should be no subtraction at this step, with subtraction = False. Mean: original, new (match down) Mean: 1.9998825 , 1.9998825 Mean: 2.99957 , 2.0058067 Mean: 4.9972 , 2.0077353 Mean: 1.9998825 , 1.9888185
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('For the method match with match_down= True, the value of the background should match the minimum of the individual backgrounds.')
print('This value should match the values in the new column above. Subtraction was done in skymatch step.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
For the method match with match_down= True, the value of the background should match the minimum of the individual backgrounds. This value should match the values in the new column above. Subtraction was done in skymatch step. Mean: 1.9980215
<matplotlib.colorbar.Colorbar at 0x7efd39ebb160>
# skymatch, match up, subtract = True
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='match', match_down=False,subtract=True,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'match' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
match_down = False # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = True # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-01-28 12:49:16,568 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-01-28 12:49:16,571 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-01-28 12:49:16,573 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-01-28 12:49:16,575 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-01-28 12:49:16,577 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-01-28 12:49:16,578 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-01-28 12:49:16,580 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-01-28 12:49:16,947 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-01-28 12:49:16,954 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': False, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-01-28 12:49:17,075 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-01-28 12:49:17,079 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-01-28 12:49:17,082 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-01-28 12:49:17,762 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-01-28 12:49:17,765 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}
2022-01-28 12:49:17,766 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-01-28 12:49:17,773 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-01-28 12:49:18,029 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-01-28 12:49:18,031 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': False, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-01-28 12:49:18,089 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:49:18,091 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-01-28 12:49:18.089597
2022-01-28 12:49:18,092 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:49:18,093 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'match'
2022-01-28 12:49:18,094 - stpipe.Image3Pipeline.skymatch - INFO - Sky matching direction: UP
2022-01-28 12:49:18,095 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: ON
2022-01-28 12:49:18,096 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:49:18,097 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing differences in sky values in overlapping regions.
2022-01-28 12:49:19,569 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: -2.98947 (old=0, delta=-2.98947)
2022-01-28 12:49:19,572 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: -1.9957 (old=0, delta=-1.9957)
2022-01-28 12:49:19,575 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0 (old=0, delta=0)
2022-01-28 12:49:19,577 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: -2.9784 (old=0, delta=-2.9784)
2022-01-28 12:49:19,578 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:49:19,579 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-01-28 12:49:19.578639
2022-01-28 12:49:19,580 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:01.489042
2022-01-28 12:49:19,581 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:49:19,896 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-01-28 12:49:20,197 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-01-28 12:49:20,480 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-01-28 12:49:20,773 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-01-28 12:49:20,774 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-01-28 12:49:21,091 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-01-28 12:49:21,094 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}
2022-01-28 12:49:21,095 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-01-28 12:49:21,101 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-01-28 12:49:21,365 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-01-28 12:49:21,368 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}
2022-01-28 12:49:21,388 - stpipe.Image3Pipeline.resample - INFO - Drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-01-28 12:49:21,527 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-01-28 12:49:22,170 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-01-28 12:49:22,887 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:23,959 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:25,054 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:26,102 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:26,465 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-01-28 12:49:27,108 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:28,065 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:28,993 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:29,934 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:30,304 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-01-28 12:49:30,884 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:31,810 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:32,737 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:33,700 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:34,064 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-01-28 12:49:34,773 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:36,057 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:37,218 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:38,547 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:38,958 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020417355 -0.018624953 0.023511130 0.016629342 359.989210484 0.019639429 359.986116709 -0.015614866
2022-01-28 12:49:39,414 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-01-28 12:49:39,415 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-01-28 12:49:39,697 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1146, 1116) from skymatch_combined_i2d.fits>,).
2022-01-28 12:49:39,700 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-01-28 12:49:39,701 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-01-28 12:49:39,704 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-01-28 12:49:39,706 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('With match_down = False, match will match the backgrounds of all to the highest background.')
print('With subtract = True, the background matching is done at the skymatch step.')
print('Mean: original, new (match up)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
With match_down = False, match will match the backgrounds of all to the highest background. With subtract = True, the background matching is done at the skymatch step. Mean: original, new (match up) Mean: 1.9998825 , 4.9893484 Mean: 2.99957 , 4.995271 Mean: 4.9972 , 4.9972 Mean: 1.9998825 , 4.978285
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('For the method match with match_down=False, the value of the background should match the maximum of the individual backgrounds')
print('This value should match the values in the new column above. Subtraction was done in skymatch step.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
For the method match with match_down=False, the value of the background should match the maximum of the individual backgrounds This value should match the values in the new column above. Subtraction was done in skymatch step. Mean: 4.9874873
<matplotlib.colorbar.Colorbar at 0x7efd39abfb80>
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'match' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = False # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-01-28 12:49:40,908 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-01-28 12:49:40,911 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-01-28 12:49:40,913 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-01-28 12:49:40,915 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-01-28 12:49:40,917 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-01-28 12:49:40,919 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-01-28 12:49:40,921 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-01-28 12:49:41,236 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-01-28 12:49:41,244 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-01-28 12:49:41,378 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-01-28 12:49:41,389 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-01-28 12:49:41,392 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-01-28 12:49:42,078 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-01-28 12:49:42,080 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}
2022-01-28 12:49:42,082 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-01-28 12:49:42,089 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-01-28 12:49:42,287 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-01-28 12:49:42,290 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-01-28 12:49:42,358 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:49:42,359 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-01-28 12:49:42.358007
2022-01-28 12:49:42,360 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:49:42,361 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'match'
2022-01-28 12:49:42,362 - stpipe.Image3Pipeline.skymatch - INFO - Sky matching direction: DOWN
2022-01-28 12:49:42,362 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: OFF
2022-01-28 12:49:42,363 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:49:42,364 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing differences in sky values in overlapping regions.
2022-01-28 12:49:43,916 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0
2022-01-28 12:49:43,918 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.993763
2022-01-28 12:49:43,919 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.98947
2022-01-28 12:49:43,920 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.0110644
2022-01-28 12:49:43,920 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:49:43,921 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-01-28 12:49:43.920824
2022-01-28 12:49:43,922 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:01.562817
2022-01-28 12:49:43,923 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:49:44,285 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-01-28 12:49:44,591 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-01-28 12:49:44,899 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-01-28 12:49:45,219 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-01-28 12:49:45,221 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-01-28 12:49:45,475 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-01-28 12:49:45,478 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}
2022-01-28 12:49:45,479 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-01-28 12:49:45,486 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-01-28 12:49:45,691 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-01-28 12:49:45,694 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}
2022-01-28 12:49:45,712 - stpipe.Image3Pipeline.resample - INFO - Drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-01-28 12:49:45,873 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-01-28 12:49:46,561 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-01-28 12:49:47,347 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:48,488 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:49,613 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:50,751 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:51,154 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-01-28 12:49:51,938 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:53,051 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:54,058 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:55,045 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:55,412 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-01-28 12:49:56,003 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:56,956 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:57,883 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:58,820 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:49:59,181 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-01-28 12:49:59,751 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:00,678 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:01,611 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:02,546 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:02,929 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020417355 -0.018624953 0.023511130 0.016629342 359.989210484 0.019639429 359.986116709 -0.015614866
2022-01-28 12:50:03,342 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-01-28 12:50:03,343 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-01-28 12:50:03,640 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1146, 1116) from skymatch_combined_i2d.fits>,).
2022-01-28 12:50:03,642 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-01-28 12:50:03,643 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-01-28 12:50:03,645 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-01-28 12:50:03,647 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('With subtract=False, no subtraction should be done here.')
print('Mean: original, new (match down)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
With subtract=False, no subtraction should be done here. Mean: original, new (match down) Mean: 1.9998825 , 1.9998825 Mean: 2.99957 , 2.99957 Mean: 4.9972 , 4.9972 Mean: 1.9998825 , 1.9998825
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('With the match method, subtract=False and match_down=True, the background value should match to the lowest individual background.')
print('This value should match the minimum values in the new column above.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
With the match method, subtract=False and match_down=True, the background value should match to the lowest individual background. This value should match the minimum values in the new column above. Mean: 1.9980215
<matplotlib.colorbar.Colorbar at 0x7efd3a755370>
The behavior of this step is that it subtracts all background. More documentation is needed on how exactly it gets to that point.
# skymatch, global+match, subtract = True
#jwst.skymatch.skymatch_step.SkyMatchStep.call(skymatch_json_file2, skymethod='global+match', match_down=True,subtract=True,output_file='MIRI',save_results=True)
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'global+match' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
#match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = True # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
#pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-01-28 12:50:05,080 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-01-28 12:50:05,083 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-01-28 12:50:05,085 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-01-28 12:50:05,087 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-01-28 12:50:05,089 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-01-28 12:50:05,091 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-01-28 12:50:05,093 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-01-28 12:50:05,448 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-01-28 12:50:05,455 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global+match', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-01-28 12:50:05,575 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-01-28 12:50:05,578 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-01-28 12:50:05,581 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-01-28 12:50:06,255 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-01-28 12:50:06,257 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}
2022-01-28 12:50:06,258 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-01-28 12:50:06,264 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-01-28 12:50:06,518 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-01-28 12:50:06,521 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global+match', 'match_down': True, 'subtract': True, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-01-28 12:50:06,571 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:06,572 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-01-28 12:50:06.571645
2022-01-28 12:50:06,573 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:06,574 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'global+match'
2022-01-28 12:50:06,574 - stpipe.Image3Pipeline.skymatch - INFO - Sky matching direction: DOWN
2022-01-28 12:50:06,575 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: ON
2022-01-28 12:50:06,575 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:06,576 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing differences in sky values in overlapping regions.
2022-01-28 12:50:08,044 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0 (old=0, delta=0)
2022-01-28 12:50:08,047 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.993763 (old=0, delta=0.993763)
2022-01-28 12:50:08,050 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.98947 (old=0, delta=2.98947)
2022-01-28 12:50:08,052 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.0110644 (old=0, delta=0.0110644)
2022-01-28 12:50:08,053 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:08,053 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing "global" sky - smallest sky value across *all* input images.
2022-01-28 12:50:08,135 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:08,136 - stpipe.Image3Pipeline.skymatch - INFO - "Global" sky value correction: 1.9658631516469498 [not converted]
2022-01-28 12:50:08,137 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:08,137 - stpipe.Image3Pipeline.skymatch - INFO - ---- Final (match+global) sky for:
2022-01-28 12:50:08,140 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.96586 (old=0, delta=1.96586)
2022-01-28 12:50:08,142 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.95963 (old=0.993763, delta=1.96586)
2022-01-28 12:50:08,144 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 4.95533 (old=2.98947, delta=1.96586)
2022-01-28 12:50:08,146 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.97693 (old=0.0110644, delta=1.96586)
2022-01-28 12:50:08,147 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:08,147 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-01-28 12:50:08.147302
2022-01-28 12:50:08,148 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:01.575657
2022-01-28 12:50:08,148 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:08,442 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-01-28 12:50:08,695 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-01-28 12:50:08,943 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-01-28 12:50:09,210 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-01-28 12:50:09,211 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-01-28 12:50:09,512 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-01-28 12:50:09,515 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}
2022-01-28 12:50:09,515 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-01-28 12:50:09,521 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-01-28 12:50:09,785 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-01-28 12:50:09,787 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}
2022-01-28 12:50:09,803 - stpipe.Image3Pipeline.resample - INFO - Drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-01-28 12:50:09,940 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-01-28 12:50:10,557 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-01-28 12:50:11,137 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:12,077 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:13,020 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:13,992 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:14,369 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-01-28 12:50:14,966 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:15,892 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:16,817 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:17,743 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:18,103 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-01-28 12:50:18,672 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:19,640 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:20,576 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:21,506 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:21,863 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-01-28 12:50:22,436 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:23,361 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:24,289 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:25,223 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:25,610 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020417355 -0.018624953 0.023511130 0.016629342 359.989210484 0.019639429 359.986116709 -0.015614866
2022-01-28 12:50:26,040 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-01-28 12:50:26,043 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-01-28 12:50:26,328 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1146, 1116) from skymatch_combined_i2d.fits>,).
2022-01-28 12:50:26,330 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-01-28 12:50:26,331 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-01-28 12:50:26,333 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-01-28 12:50:26,334 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('Mean: original, new (global+match)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
Mean: original, new (global+match) Mean: 1.9998825 , 0.034019697 Mean: 2.99957 , 0.039943673 Mean: 4.9972 , 0.041871887 Mean: 1.9998825 , 0.022955226
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
#print('Did resample subtract a background level, and what was it?')
print('This value should match the values in the new column above. Subtraction was done in skymatch step.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
This value should match the values in the new column above. Subtraction was done in skymatch step. Mean: 0.032158297
<matplotlib.colorbar.Colorbar at 0x7efd391dfdf0>
# Test running just skymatch and resample and skiping other steps.
# Run Calwebb_image3 on the association table
# set any specific parameters
skymethod = 'global+match' # sky computation algorithm to be used. Allowed values: {local, global, match, global+match} (Default = global+match
#match_down = True # Specifies whether the sky differences should be subtracted from images with higher sky values (match_down = True) in order to match the image with the lowest sky or sky differences should be added to the images with lower sky values to match the sky of the image with the highest sky value (match_down = False). (Default = True
subtract = False # Specifies whether the computed sky background values are to be subtracted from the images. (Default = False
pipe3=Image3Pipeline()
pipe3.tweakreg.skip = True
pipe3.outlier_detection.skip = True
pipe3.source_catalog.skip = True
pipe3.skymatch.skymethod = skymethod
#pipe3.skymatch.match_down = match_down
pipe3.skymatch.subtract = subtract
pipe3.skymatch.output_file = 'MIRI'
pipe3.skymatch.save_results = True
pipe3.resample.save_results = True
pipe3.save_results = True
# run Image3
image = pipe3.run(skymatch_json_file2)
print('Image 3 pipeline finished.')
2022-01-28 12:50:27,776 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.
2022-01-28 12:50:27,777 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.
2022-01-28 12:50:27,779 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.
2022-01-28 12:50:27,781 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.
2022-01-28 12:50:27,783 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.
2022-01-28 12:50:27,784 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.
2022-01-28 12:50:27,786 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.
2022-01-28 12:50:28,060 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('skymatch_asnfile2.json',).
2022-01-28 12:50:28,067 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global+match', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}
2022-01-28 12:50:28,185 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits' reftypes = ['drizpars']
2022-01-28 12:50:28,187 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.
2022-01-28 12:50:28,190 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...
2022-01-28 12:50:28,758 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).
2022-01-28 12:50:28,761 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'brightest': 1000, 'peakmax': None, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 1.0, 'use2dhist': True, 'separation': 0.5, 'tolerance': 1.0, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'general', 'nclip': 3, 'sigma': 3.0, 'align_to_gaia': False, 'gaia_catalog': 'GAIADR2', 'min_gaia': 5, 'save_gaia_catalog': False}
2022-01-28 12:50:28,762 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.
2022-01-28 12:50:28,768 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done
2022-01-28 12:50:28,905 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).
2022-01-28 12:50:28,907 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope@2/tmp/tmpysdlyldj/MIRI', 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'global+match', 'match_down': True, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '0', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}
2022-01-28 12:50:28,959 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:28,960 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() started on 2022-01-28 12:50:28.959249
2022-01-28 12:50:28,961 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:28,961 - stpipe.Image3Pipeline.skymatch - INFO - Sky computation method: 'global+match'
2022-01-28 12:50:28,962 - stpipe.Image3Pipeline.skymatch - INFO - Sky matching direction: DOWN
2022-01-28 12:50:28,963 - stpipe.Image3Pipeline.skymatch - INFO - Sky subtraction from image data: OFF
2022-01-28 12:50:28,963 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:28,964 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing differences in sky values in overlapping regions.
2022-01-28 12:50:30,611 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0
2022-01-28 12:50:30,612 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.993763
2022-01-28 12:50:30,613 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.98947
2022-01-28 12:50:30,613 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 0.0110644
2022-01-28 12:50:30,614 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:30,614 - stpipe.Image3Pipeline.skymatch - INFO - ---- Computing "global" sky - smallest sky value across *all* input images.
2022-01-28 12:50:30,693 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:30,694 - stpipe.Image3Pipeline.skymatch - INFO - "Global" sky value correction: 1.9658632242131073 [not converted]
2022-01-28 12:50:30,695 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:30,696 - stpipe.Image3Pipeline.skymatch - INFO - ---- Final (match+global) sky for:
2022-01-28 12:50:30,696 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq1_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.96586 (old=0, delta=1.96586)
2022-01-28 12:50:30,697 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq2_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 2.95963 (old=0.993763, delta=1.96586)
2022-01-28 12:50:30,697 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq3_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 4.95533 (old=2.98947, delta=1.96586)
2022-01-28 12:50:30,698 - stpipe.Image3Pipeline.skymatch - INFO - * Image ID=det_image_seq4_MIRIMAGE_F560Wexp1_cal_skymatch.fits. Sky background: 1.97693 (old=0.0110644, delta=1.96586)
2022-01-28 12:50:30,698 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:30,699 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() ended on 2022-01-28 12:50:30.698901
2022-01-28 12:50:30,700 - stpipe.Image3Pipeline.skymatch - INFO - ***** jwst.skymatch.skymatch.match() TOTAL RUN TIME: 0:00:01.739652
2022-01-28 12:50:30,700 - stpipe.Image3Pipeline.skymatch - INFO -
2022-01-28 12:50:30,987 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_0_skymatch.fits
2022-01-28 12:50:31,233 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_1_skymatch.fits
2022-01-28 12:50:31,482 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_2_skymatch.fits
2022-01-28 12:50:31,743 - stpipe.Image3Pipeline.skymatch - INFO - Saved model in MIRI_3_skymatch.fits
2022-01-28 12:50:31,744 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done
2022-01-28 12:50:31,923 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).
2022-01-28 12:50:31,926 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'crf', 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None}
2022-01-28 12:50:31,926 - stpipe.Image3Pipeline.outlier_detection - INFO - Step skipped.
2022-01-28 12:50:31,933 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done
2022-01-28 12:50:32,079 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).
2022-01-28 12:50:32,081 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'pixel_scale_ratio': 1.0, 'single': False, 'blendheaders': True, 'allowed_memory': None}
2022-01-28 12:50:32,096 - stpipe.Image3Pipeline.resample - INFO - Drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits
2022-01-28 12:50:32,232 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for skymatch_combined.fits
2022-01-28 12:50:32,860 - stpipe.Image3Pipeline.resample - INFO - Resampling science data
2022-01-28 12:50:33,435 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:34,368 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:35,292 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:36,221 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:36,585 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise
2022-01-28 12:50:37,168 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:38,094 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:39,027 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:39,956 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:40,317 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson
2022-01-28 12:50:40,886 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:41,813 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:42,735 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:43,659 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:44,018 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat
2022-01-28 12:50:44,584 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:45,518 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:46,441 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:47,365 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1146, 1116)
2022-01-28 12:50:47,746 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS 0.020417355 -0.018624953 0.023511130 0.016629342 359.989210484 0.019639429 359.986116709 -0.015614866
2022-01-28 12:50:48,165 - stpipe.Image3Pipeline.resample - INFO - Saved model in skymatch_combined_i2d.fits
2022-01-28 12:50:48,166 - stpipe.Image3Pipeline.resample - INFO - Step resample done
2022-01-28 12:50:48,342 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1146, 1116) from skymatch_combined_i2d.fits>,).
2022-01-28 12:50:48,344 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': True, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 5, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}
2022-01-28 12:50:48,344 - stpipe.Image3Pipeline.source_catalog - INFO - Step skipped.
2022-01-28 12:50:48,347 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done
2022-01-28 12:50:48,348 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done
Image 3 pipeline finished.
img01_1_new = datamodels.open('MIRI_0_skymatch.fits')
img01_2_new = datamodels.open('MIRI_1_skymatch.fits')
img02_1_new = datamodels.open('MIRI_2_skymatch.fits')
img02_2_new = datamodels.open('MIRI_3_skymatch.fits')
print('Mean: original, new (global+match)')
print('Mean:',np.mean(img01_1.data),',', np.mean(img01_1_new.data))
print('Mean:',np.mean(img01_2.data),',', np.mean(img01_2_new.data))
print('Mean:',np.mean(img02_1.data),',', np.mean(img02_1_new.data))
print('Mean:',np.mean(img02_2.data),',', np.mean(img02_2_new.data))
Mean: original, new (global+match) Mean: 1.9998825 , 1.9998825 Mean: 2.99957 , 2.99957 Mean: 4.9972 , 4.9972 Mean: 1.9998825 , 1.9998825
# Look at combined image
im_file = ImageModel('skymatch_combined_i2d.fits')
print('Did resample subtract a background level, and what was it?')
#print('This value should match the values in the new column above. Subtraction was done in skymatch step.')
print('Mean:',np.mean(im_file.data[600:700,600:700]))
# read in ecsv photom file
from astropy.visualization import LogStretch, PercentileInterval, ManualInterval
#viz2 = LogStretch() + ManualInterval(0,10)
plt.figure(figsize=(20,20))
#plt.imshow(viz2(im_file.data), origin='lower')
plt.imshow(im_file.data, origin='lower', cmap='rainbow', vmin=0, vmax=4)
plt.colorbar()
Did resample subtract a background level, and what was it? Mean: 0.032158192
<matplotlib.colorbar.Colorbar at 0x7efd3910aee0>
